{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 准备工作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "fn = { \"output\" : { \"公众号_htm_snippets\": \"data_raw_src/公众号_htm_snippets_{公众号}.tsv\",\n",
    "                    \"公众号_df\": \"data_raw_src/公众号_df_{公众号}.tsv\",\n",
    "                    \"公众号_xlsx\": \"data_sets/公众号_url_{公众号}.xlsx\" } \\\n",
    "      }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 采集公众号（requests）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "5\n",
      "10\n",
      "15\n",
      "20\n",
      "25\n",
      "30\n",
      "35\n",
      "40\n",
      "45\n",
      "50\n",
      "55\n",
      "60\n",
      "65\n",
      "70\n",
      "75\n",
      "80\n",
      "85\n",
      "90\n",
      "95\n",
      "100\n",
      "105\n",
      "110\n",
      "115\n",
      "120\n",
      "125\n",
      "130\n",
      "135\n",
      "140\n",
      "145\n",
      "150\n",
      "155\n",
      "160\n",
      "165\n",
      "170\n",
      "175\n",
      "180\n",
      "185\n",
      "190\n",
      "195\n",
      "200\n",
      "205\n",
      "210\n",
      "215\n",
      "220\n",
      "225\n",
      "230\n",
      "235\n",
      "240\n",
      "245\n"
     ]
    }
   ],
   "source": [
    "# 目标url\n",
    "\n",
    "import time\n",
    "import requests\n",
    "import pandas as pd\n",
    "import csv\n",
    "\n",
    "\n",
    "url = \"https://mp.weixin.qq.com/cgi-bin/appmsg\"\n",
    "\n",
    "# 使用Cookie，跳过登陆操作\n",
    "headers = {\n",
    "    \"Cookie\":\"pgv_pvid=5999288730; pgv_pvi=7422254080; RK=k9hQPEIzXP; ptcz=8b93409e5d1cee3105a998a4bbbc48e7393e1923c734681fe0d014c6bb431136; luin=o1764569742; tvfe_boss_uuid=607a130922240cd8; o_cookie=1764569742; ua_id=Zia1PSczHHIZsO8bAAAAAHEFPg5-3ENFAfYtBxJuRtM=; openid2ticket_oC09X45uW7vYTtXD898hD7mdASYs=ap37uWhuto9dNyRYdtbNKgwph1niKoRk772iOl95ZoM=; mm_lang=zh_CN; lskey=00010000ffde29d43e01a7756ac87cd95ce59d8eee57b5b76c0b1ac2e5c65afd6e895b9d53dcbddb919d55d5; wxuin=89547329189924; openid2ticket_o5pks5Pw3Ex7MZeqKUv-N2ReqYrA=jQXjSgptNgegmLDNBVCUm5phGVPSO2lMq95Ul/K1Ujc=; rand_info=CAESIEDN2PoDQICTa9VqTFCnQneVmCli9KnfTovNRJvv7gSr; slave_bizuin=3285243811; data_bizuin=3285243811; bizuin=3285243811; data_ticket=o+SlESeKHpovP/C9Tj8ygDIijZ8P1yh1ShLNSDBr9k405/iRb3qF33ksj2Y18l7g; slave_sid=RDJTXzdIUWZQaVpnVDkwcnZLbG0yQTl5UVZKUzZBV2VTaEpKZGVKMkc2bkdvZ2J5eWtJR0hvcHdFcXRvTUtLWlBNYXA1SHZ5QmxtalpGTGcxMWwxVGJiclFPRHp5MnpjODdlX3pvb1pBZ2FIbWIxbnFobkdKWFFQQTdIdTVTZGRUam1mQ0tvWGZGVUFNNlNR; slave_user=gh_bcfb4539b17b; xid=b1cf8f96d3de9ba5102465ce2a1d48bf; openid2ticket_oo9fQw84KKQabDNa6n49jsW8GTxU=KIChWpj0Ey/LforhUQXclIqbvF3TQ8mcAwFPqH05H98=\",\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36\"}\n",
    "\n",
    "data = {\n",
    "    \"token\": \"1290050666\",     # 各个用户唯一的\n",
    "    \"lang\": \"zh_CN\",\n",
    "    \"f\": \"json\",\n",
    "    \"ajax\": \"1\",\n",
    "    \"action\": \"list_ex\",\n",
    "    \"begin\": \"0\",\n",
    "    \"count\": \"5\",\n",
    "    \"query\": \"\",\n",
    "    \"fakeid\": \"MzU1ODQ0NjAwMQ==\",   # 各个微信公众号的各自独有的ID\n",
    "    \"type\": \"9\",\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "content_list=[]\n",
    "\n",
    "for i in range(50):    # 运行5页\n",
    "    data[\"begin\"] = i*5\n",
    "    print(data[\"begin\"])\n",
    "    time.sleep(3)     # 3秒一次\n",
    "    # 使用get方法进行提交\n",
    "    content_json = requests.get(url, headers=headers, params=data).json()\n",
    "#     print(content_json)\n",
    "    # 返回了一个json，里面是每一页的数据\n",
    "    \n",
    "    for item in content_json[\"app_msg_list\"]:\n",
    "    # 提取每页文章的标题及对应的url\n",
    "        items = []\n",
    "        items.append(item[\"title\"])   # 标题\n",
    "        items.append(item[\"link\"])     #  链接\n",
    "        items.append(item[\"create_time\"])     # 发布时间\n",
    "        content_list.append(items)\n",
    "\n",
    "\n",
    "name=['title','link','create_time']\n",
    "test=pd.DataFrame(columns=name,data=content_list)\n",
    "with pd.ExcelWriter(fn[\"output\"][\"公众号_xlsx\"].format(公众号=\"推理大师Club_requests_text01\")) as writer:\n",
    "    test.to_excel(writer)\n",
    "\n",
    "# test.to_csv(\"../微信公众号爬虫_zhichao/南方周末.csv\",mode='a',encoding='utf-8')\n",
    "# print(\"保存成功\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
